#include "Probe.h"
#include "CanonicalConv.h"
#include "LinkParser.h"
#include "HTMLFetcher.h"
#include "RobotVerifier.h"
#include "Site.h"
#include "Domain.h"
#include <string>
#include <iostream>
#include <time.h>
#include <exception>
#include <stdexcept>
#include "md5/md5.h"
#include <pthread.h>
#include "LogProbe.h"
#include <sstream>
#include <csignal>

//#define LOG

using namespace std;

NCManager* Probe::spaceship;

Probe::Probe()
{
}

Probe::~Probe()
{
}

void Probe::setMotherShip(NCManager* ship)
{
	Probe::spaceship = ship;
}
void* Probe::run(void * threadID)
{
	signal (SIGPIPE, SIG_IGN);

#ifdef LOG
	string filename("Probe");
	stringstream lStream; 
	lStream << threadID;
	filename += lStream.str();
	cout << "Filename: "<< filename << endl;
	LogProbe *log;
	log=  new LogProbe(filename);
#endif
	HTMLFetcher* fetcher = new HTMLFetcher();
	//int sitesFetched = 0;
	bool aux = false;
	while(true)
	{
		#ifdef LOG	
		log->GravarLog("Craw a new site");		
		log->SetTime();
		#endif
		string urlToVisit ="?";
		pthread_mutex_lock(&Probe::spaceship->queueMutex);
		urlToVisit = Probe::spaceship->popUrlToVisit();
		pthread_mutex_unlock(&Probe::spaceship->queueMutex);
		while(urlToVisit == "?") {
		pthread_mutex_lock(&Probe::spaceship->queueMutex);
		urlToVisit = Probe::spaceship->popUrlToVisit();
		cout << "ZZZZZZZZZzzzzzzzzzzz" << endl;
		pthread_mutex_unlock(&Probe::spaceship->queueMutex);
		sleep(2);
		}
		#ifdef LOG
		log->GravarLog("PopUrlToVisit: ",log->GetDeltaTime());
		#endif
		
		//cout << " #" << (int)threadID << " Just grabbed URL: " << urlToVisit << endl;		
		string urlDomain = CanonicalConv::getCanonicalDomain(urlToVisit);		
		pthread_mutex_lock(&Probe::spaceship->domainsMutex);
		aux = !Probe::spaceship->hasDomain(urlDomain);
		pthread_mutex_unlock(&Probe::spaceship->domainsMutex);
		
		#ifdef LOG
		string s = "Just grabbed URL";
		s += urlToVisit;		
		log->GravarLog(s);
		#endif
		if(aux)
		{
			#ifdef LOG
			log->SetTime();
			#endif
			Domain* d = new Domain(urlDomain);
			pthread_mutex_lock(&Probe::spaceship->domainsMutex);
			Probe::spaceship->addDomain(d);
			pthread_mutex_unlock(&Probe::spaceship->domainsMutex);
			#ifdef LOG
			log->GravarLog("Add Domain ",log->GetTime());
			#endif
		}
		#ifdef LOG
		log->SetTime();
		#endif
		pthread_mutex_lock(&Probe::spaceship->donwloadedSitesMutex);
		aux = !Probe::spaceship->isDownloaded(urlToVisit);
		pthread_mutex_unlock(&Probe::spaceship->donwloadedSitesMutex);
		#ifdef LOG
		log->GravarLog("Get Domain ",log->GetDeltaTime() );
		#endif
		if(aux)
		{
			time_t now;
			time(&now);
			#ifdef LOG
			log->SetTime();
			#endif
			pthread_mutex_lock(&Probe::spaceship->domainsMutex);
			aux = (now >= Probe::spaceship->getDomain(urlDomain)->getWhenAvailable());
			pthread_mutex_unlock(&Probe::spaceship->domainsMutex);
			#ifdef LOG
			log->GravarLog("getWhenAvailable ",log->GetDeltaTime() );
			#endif
			if(aux)
			{
				
				#ifdef LOG
				log->SetTime();
				#endif
				bool robots;		
				pthread_mutex_lock(&Probe::spaceship->domainsMutex);
					robots = Probe::spaceship->getDomain(urlDomain)->getFetchRobots();				 	
				pthread_mutex_unlock(&Probe::spaceship->domainsMutex);
				#ifdef LOG
				log->GravarLog("getFetchRobots ",log->GetDeltaTime() );
				#endif

				if(!robots)
				{
					#ifdef LOG
					log->SetTime();
					#endif
					int resp = RobotVerifier::fetchRobotsTXT(Domain::getTrueDomain(urlToVisit));
					
					pthread_mutex_lock(&Probe::spaceship->domainsMutex);
							Probe::spaceship->getDomain(urlDomain)->setFetchRobots();
					pthread_mutex_unlock(&Probe::spaceship->domainsMutex);
					#ifdef LOG
					log->GravarLog("setFetchRobots ",log->GetDeltaTime() );
					log->SetTime();
					#endif
					if (resp == 0)
					{
						pthread_mutex_lock(&Probe::spaceship->domainsMutex);
						 	Probe::spaceship->getDomain(urlDomain)->setHasRobots();					 	
						pthread_mutex_unlock(&Probe::spaceship->domainsMutex);
						pthread_mutex_lock(&Probe::spaceship->robotsCounterMutex);
						NCManager::fetchedRobots++;
						pthread_mutex_unlock(&Probe::spaceship->robotsCounterMutex);
						
					}
					#ifdef LOG
					log->GravarLog("setHasRobots ",log->GetDeltaTime() );
					#endif
				}
				bool hasRobots = false;
				bool mayCrawl  = false;
				#ifdef LOG
				log->SetTime();
				#endif
				pthread_mutex_lock(&Probe::spaceship->domainsMutex);
					hasRobots = Probe::spaceship->getDomain(urlDomain)->getHasRobots();
				pthread_mutex_unlock(&Probe::spaceship->domainsMutex);
				#ifdef LOG
				log->GravarLog("getHasRobots ",log->GetDeltaTime() );
				#endif				
				if(hasRobots)
					mayCrawl = RobotVerifier::analyseRobotsTXT(urlToVisit,Domain::getTrueDomain(urlToVisit));
				
				if(!hasRobots || (mayCrawl && hasRobots))
				{
					
					char* temp = const_cast<char*>(urlToVisit.c_str()) ;
					try{
						#ifdef LOG
						log->SetTime();
						#endif
						fetcher->fetchPage(urlToVisit,MD5String(temp));
						//cout << "Probe D/L:  " << urlToVisit << endl;
						now = time(&now);
						#ifdef LOG
						log->GravarLog("fetchPage ",log->GetDeltaTime() );
						log->SetTime();
						#endif	
											//cout << " #" << (int)threadID << "Setting Domain(whenAvailable & addSite)" << endl;
											pthread_mutex_lock(&Probe::spaceship->domainsMutex);
											Probe::spaceship->getDomain(urlDomain)->setWhenAvailable(now+30);
											pthread_mutex_unlock(&Probe::spaceship->domainsMutex);
											pthread_mutex_lock(&Probe::spaceship->donwloadedSitesMutex);
											Probe::spaceship->addToDownloadedSet(urlToVisit);
											pthread_mutex_unlock(&Probe::spaceship->donwloadedSitesMutex);
						#ifdef LOG
						log->GravarLog("Adding Site at Domain ",log->GetDeltaTime() );
						#endif										
						//sitesFetched++;
						pthread_mutex_lock(&Probe::spaceship->pageCounterMutex);
						NCManager::visitedSites++;
						pthread_mutex_unlock(&Probe::spaceship->pageCounterMutex);
						#ifdef LOG
						log->SetTime();
						#endif
						LinkParser::getLinksFromFile(Probe::spaceship,MD5String(temp),urlToVisit);
						#ifdef LOG
						log->GravarLog("Get Links For File ",log->GetDeltaTime() );
						#endif
					}
					catch(...)
					{
						pthread_mutex_unlock(&Probe::spaceship->queueMutex);
						#ifdef LOG
						log->GravarLog("Erro no Fetch da pagina");
						#endif
					}
					
				}				
		
			}
			else
			{
			    #ifdef LOG
				log->SetTime();
				#endif
				//pthread_mutex_lock(&Probe::spaceship->domainsMutex);
				//cout << " Domain: " << urlDomain << " has been visited lately, missing: " << (Probe::spaceship->getDomain(urlDomain)->getWhenAvailable() - now)<< endl;
				//pthread_mutex_unlock(&Probe::spaceship->domainsMutex);
				#ifdef LOG
				s = "Domain";
				s += urlDomain;
				s += " has been visited lately";
				log->GravarLog(s,log->GetDeltaTime() );
				log->SetTime();
				#endif
				
				pthread_mutex_lock(&Probe::spaceship->queueMutex);
				Probe::spaceship->addToVisitQueue(urlToVisit);
				pthread_mutex_unlock(&Probe::spaceship->queueMutex);
				#ifdef LOG
				log->GravarLog("addToVisitQueue AGAIN" ,log->GetDeltaTime() );
				#endif
			}
		}		
	}
	delete(fetcher);
#ifdef LOG
	delete(log);
#endif
	pthread_exit(0);
}
